Version: 1.2 (Jupytext, time measurements, logger, param notebook execution, fixes)
Sample data sets for testing anomaly detection pipelines and algorithms.
General settings:
abs_values - If values are bigger or equal to zero.add_zeros - If to add zeros. This is especially useful together with abs_value for gesting logaritmic.In general, the data set has to have following structure:
ATTR_DATE_TIME. Datetime index as the index column.ATTR_ID as an integer identifier of the observation. This is necessary for cross validation and models and pipelines assessment for better work than with data time format.For more information, see notebook notebooks/documentation/anomaly_detection_sample_data_documentation.
import sys
import os
sys.path+=[os.path.join(os.getcwd(), ".."), os.path.join(os.getcwd(), "../..")] # one and two up
ToC
Necessary libraries for notebook functionality:
NOTE: This way, using the function, the button works only in active notebook. If the functionality needs to be preserved in html export, then the code has to be incluced directly into notebook.
LOGGER_CONFIG_NAME = "logger_file_limit_console"
ADDAPT_WIDTH = False
try:
from src.utils.notebook_support_functions import create_button, get_notebook_name
NOTEBOOK_NAME = get_notebook_name()
SUPPORT_FUNCTIONS_READ = True
except:
NOTEBOOK_NAME = "NO_NAME"
SUPPORT_FUNCTIONS_READ = False
from src.utils.logger import Logger
from src.utils.envs import Envs
from src.utils.config import Config
from pandas import options
from IPython.display import display, HTML
options.display.max_rows = 500
options.display.max_columns = 500
envs = Envs()
envs.set_logger(LOGGER_CONFIG_NAME)
Logger().start_timer(f"NOTEBOOK; Notebook name: {NOTEBOOK_NAME}")
if ADDAPT_WIDTH:
display(HTML("<style>.container { width:100% !important; }</style>")) # notebook width
2024-02-08 10:04:43,796 - file_limit_console - INFO - Logger was created on WS-3000 in branche 019_unify_pipelines_inputs. 2024-02-08 10:04:43,798 - file_limit_console - INFO - Process: NOTEBOOK; Notebook name: anomaly_detection_sample_data_execution.py; Timer started;
# create_button()
from datetime import datetime
from src.utils.date_time_functions import create_datetime_id
from src.data.anomaly_detection_sample_data import AnomalyDetectionSampleData, plot_data_frame_series
from src.data.attributes import A
from src.data.anomaly_detection_sample_data import ATTRS
# from src.global_constants import * # Remember to import only the constants in use
N_ROWS_TO_DISPLAY = 2
FIGURE_SIZE_SETTING = {"autosize": False, "width": 2200, "height": 750}
DATA_PROCESSING_CONFIG_NAME = "data_processing_basic"
# MANDATORY FOR CONFIG DEFINITION AND NOTEBOOK AND ITS OUTPUTS IDENTIFICATION #########################################
PYTHON_CONFIG_NAME = "python_local"
ID = create_datetime_id(now=datetime.now(), add_micro=False)
# (END) MANDATORY FOR CONFIG DEFINITION AND NOTEBOOK AND ITS OUTPUTS IDENTIFICATION ###################################
envs.set_config(PYTHON_CONFIG_NAME)
data = AnomalyDetectionSampleData(abs_values=False, add_zeros=False)
df = data.generate(n=10, seed_number=376)
df.head()
| DATE_TIME | ID | ATTR_1 | ATTR_2 | ATTR_3 | ATTR_4 | ATTR_5 | |
|---|---|---|---|---|---|---|---|
| 1980-01-01 | 1980-01-01 | 0 | 0.205113 | 7.056803 | 0.034343 | 6.866656 | 3.567969 |
| 1980-01-02 | 1980-01-02 | 1 | -0.413252 | 1.321864 | 0.172800 | -0.370840 | 5.822740 |
| 1980-01-03 | 1980-01-03 | 2 | 1.759583 | 2.456098 | 0.613517 | 2.578916 | 4.703235 |
| 1980-01-04 | 1980-01-04 | 3 | 0.582578 | 3.767234 | 0.794213 | 5.803167 | 4.702401 |
| 1980-01-05 | 1980-01-05 | 4 | 1.001934 | 1.689356 | -1.208928 | 8.764840 | 4.150775 |
plot_data_frame_series(df, ATTRS)
Logger().end_timer()
2024-02-08 10:04:44,680 - file_limit_console - INFO - Process: NOTEBOOK; Notebook name: anomaly_detection_sample_data_execution.py; Timer ended; Process Duration [s]: 0.88; Process Duration [m]: 0.01